{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\h5py\\__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n"
     ]
    }
   ],
   "source": [
    "import gym\n",
    "import os\n",
    "import sys\n",
    "import itertools\n",
    "import numpy as np\n",
    "import random\n",
    "import tensorflow as tf\n",
    "from collections import defaultdict, namedtuple\n",
    "\n",
    "import matplotlib\n",
    "from matplotlib import pyplot as plt\n",
    "%matplotlib inline\n",
    "matplotlib.style.use('ggplot')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Deep q Network\n",
    "flags.DEFINE_boolean('use_gpu', True, 'Whether to use gpu or not. gpu use NHWC and gpu use NCHW for data_format')\n",
    "flags.DEFINE_string('agent_type', 'DQN', 'The type of agent [DQN]')\n",
    "flags.DEFINE_boolean('double_q', False, 'Whether to use double Q-learning')\n",
    "flags.DEFINE_string('network_header_type', 'nips', 'The type of network header [mlp, nature, nips]')\n",
    "flags.DEFINE_string('network_output_type', 'normal', 'The type of network output [normal, dueling]')\n",
    "\n",
    "# Environment\n",
    "flags.DEFINE_string('env_name', 'Breakout-v0', 'The name of gym environment to use')\n",
    "flags.DEFINE_integer('n_action_repeat', 1, 'The number of actions to repeat')\n",
    "flags.DEFINE_integer('max_random_start', 30, 'The maximum number of NOOP actions at the beginning of an episode')\n",
    "flags.DEFINE_integer('history_length', 4, 'The length of history of observation to use as an input to DQN')\n",
    "flags.DEFINE_integer('max_r', +1, 'The maximum value of clipped reward')\n",
    "flags.DEFINE_integer('min_r', -1, 'The minimum value of clipped reward')\n",
    "flags.DEFINE_string('observation_dims', '[80, 80]', 'The dimension of gym observation')\n",
    "flags.DEFINE_boolean('random_start', True, 'Whether to start with random state')\n",
    "flags.DEFINE_boolean('use_cumulated_reward', False, 'Whether to use cumulated reward or not')\n",
    "\n",
    "# Training\n",
    "flags.DEFINE_boolean('is_train', True, 'Whether to do training or testing')\n",
    "flags.DEFINE_integer('max_delta', None, 'The maximum value of delta')\n",
    "flags.DEFINE_integer('min_delta', None, 'The minimum value of delta')\n",
    "flags.DEFINE_float('ep_start', 1., 'The value of epsilon at start in e-greedy')\n",
    "flags.DEFINE_float('ep_end', 0.01, 'The value of epsilnon at the end in e-greedy')\n",
    "flags.DEFINE_integer('batch_size', 32, 'The size of batch for minibatch training')\n",
    "flags.DEFINE_integer('max_grad_norm', None, 'The maximum norm of gradient while updating')\n",
    "flags.DEFINE_float('discount_r', 0.99, 'The discount factor for reward')\n",
    "\n",
    "# Timer\n",
    "flags.DEFINE_integer('t_train_freq', 4, '')\n",
    "\n",
    "# Below numbers will be multiplied by scale\n",
    "flags.DEFINE_integer('scale', 10000, 'The scale for big numbers')\n",
    "flags.DEFINE_integer('memory_size', 100, 'The size of experience memory (*= scale)')\n",
    "flags.DEFINE_integer('t_target_q_update_freq', 1, 'The frequency of target network to be updated (*= scale)')\n",
    "flags.DEFINE_integer('t_test', 1, 'The maximum number of t while training (*= scale)')\n",
    "flags.DEFINE_integer('t_ep_end', 100, 'The time when epsilon reach ep_end (*= scale)')\n",
    "flags.DEFINE_integer('t_train_max', 5000, 'The maximum number of t while training (*= scale)')\n",
    "flags.DEFINE_float('t_learn_start', 5, 'The time when to begin training (*= scale)')\n",
    "flags.DEFINE_float('learning_rate_decay_step', 5, 'The learning rate of training (*= scale)')\n",
    "\n",
    "# Optimizer\n",
    "flags.DEFINE_float('learning_rate', 0.00025, 'The learning rate of training')\n",
    "flags.DEFINE_float('learning_rate_minimum', 0.00025, 'The minimum learning rate of training')\n",
    "flags.DEFINE_float('learning_rate_decay', 0.96, 'The decay of learning rate of training')\n",
    "flags.DEFINE_float('decay', 0.99, 'Decay of RMSProp optimizer')\n",
    "flags.DEFINE_float('momentum', 0.0, 'Momentum of RMSProp optimizer')\n",
    "flags.DEFINE_float('gamma', 0.99, 'Discount factor of return')\n",
    "flags.DEFINE_float('beta', 0.01, 'Beta of RMSProp optimizer')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "FLAGS.__delattr__() \n",
    "\n",
    "flags = tf.app.flags\n",
    "FLAGS = flags.FLAGS\n",
    "\n",
    "flags.DEFINE_boolean(\"duele\", False, \"use dueling deep Q-learning\")\n",
    "flags.DEFINE_boolean(\"double\", False, \"use double Q-learning\")\n",
    "\n",
    "flags.DEFINE_boolean(\"is_train\", True, \"training or testing\")\n",
    "flags.DEFINE_integer(\"random_seed\", 123, \"value of random seed\")\n",
    "flags.DEFINE_boolean(\"display\", False, \"display the game\")\n",
    "flags.DEFINE_integer(\"scale\", 10000, \"step and the memory size\")\n",
    "flags.DEFINE_integer(\"batch_size\", 32, \"batch size\")\n",
    "flags.DEFINE_float(\"discount\", 0.99)\n",
    "flags.DEFINE_float(\"learning_rate\", 0.00025)\n",
    "flags.DEFINE_float(\"learning_rate_min\", 0.00025)\n",
    "flags.DEFINE_float(\"learning_rate_decay\", 0.96)\n",
    "flags.DEFINE_integer(\"history_length\", 4)\n",
    "flags.DEFINE_integer(\"train_frequency\", 4)\n",
    "\n",
    "flags.DEFINE_integer(\"learn_start\", 50000)\n",
    "flags.DEFINE_integer(\"frame_width\", 84)\n",
    "flags.DEFINE_integer(\"frame_height\", 84)\n",
    "flags.DEFINE_integer(\"max_reward\", 1)\n",
    "flags.DEFINE_integer(\"min_reward\", -1)\n",
    "flags.DEFINE_integer(\"episode_in_test\", 80)\n",
    "flags.DEFINE_integer(\"episode_in_train\", 18000)\n",
    "flags.DEFINE_integer(\"test_max_step\", 10000)\n",
    "\n",
    "FLAGS = flags.FLAGS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "env = gym.make('Breakout-v0')\n",
    "env = env.unwrapped\n",
    "env.seed(FLAGS.random_seed)\n",
    "tf.set_random_seed(FLAGS.random_seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Environment(object):\n",
    "    def __init__(self, env, history):\n",
    "        self.env = env\n",
    "        self.reward = 0\n",
    "        self.terminal = False\n",
    "        self.state_history = history\n",
    "        self.state_dim = (FLAGS.frame_width, FLAGS.frame_height)\n",
    "        self.nA = self.env.action_space.n\n",
    "        self.nS = None\n",
    "\n",
    "    def reset(self):\n",
    "        self.env.reset()\n",
    "    \n",
    "    def random_start(self):\n",
    "        self.reset()\n",
    "        for _ in reversed(range(random.randint(4, 30))):\n",
    "            state, _, _, _ = self.env.step(0)\n",
    "            if 4 - _ > 0:\n",
    "                self.state_history.push(self.__frame(state))\n",
    "        \n",
    "        self.env.render()\n",
    "        return self.state_history\n",
    "    \n",
    "    def step(self, action):\n",
    "        state, self.reward, self.terminal, _ = self.env.step(action)\n",
    "        self.state = self.__frame(state)\n",
    "\n",
    "        self.env.render()\n",
    "        return self.state, self.reward, self.terminal\n",
    "    \n",
    "    @property\n",
    "    def __frame(self, state):\n",
    "        processed_state = np.array(state)\n",
    "        frame_state = np.uint8(resize(rgb2gray(processed_state)/255., self.state_dim))\n",
    "        return frame_state"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class History(object):\n",
    "    def __init__(self):\n",
    "        self.history = np.zeros([FLAGS.history_length, \n",
    "                                FLAGS.frame_width,\n",
    "                                FLAGS.frame_height], dtype=np.float32)\n",
    "        \n",
    "    def push(self, state):\n",
    "        self.history[:-1] = self.history[1:]\n",
    "        self.history[-1] = state\n",
    "        \n",
    "    def get():\n",
    "        retrun self.history\n",
    "        \n",
    "    def clean():\n",
    "        self.history *= 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Memory(object):\n",
    "    def __init__(self):\n",
    "        pass\n",
    "    \n",
    "    def push(self):\n",
    "        pass\n",
    "    \n",
    "    def getState(self):\n",
    "        pass\n",
    "    \n",
    "    def sample(self):\n",
    "        pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Agent(object):\n",
    "    def __init__(self, env, history, memory):\n",
    "        self.env = env\n",
    "        self.nA = env.nA\n",
    "        self.state_history = history\n",
    "        self.state_memroy = memory\n",
    "        self.t = 0\n",
    "        \n",
    "        self.q_value, self.q_network = self.__build_network()\n",
    "        self.target_q_value, self.target_q_network = self.__build_network()\n",
    "        \n",
    "        self.sess = tf.Session()\n",
    "        \n",
    "        tf.summary.FileWriter(\"summary/\", self.sess.graph)\n",
    "        self.sess.run(tf.global_variables_initializer())\n",
    "        self.saver = tf.train.Saver()\n",
    "    \n",
    "    def predict(self, state):\n",
    "        if self.t < FLAGS.learn_start:\n",
    "            action = random.randrange(self.nA)\n",
    "        else:\n",
    "            action = np.argmax(self.q_value.eval(feed_dict={self.s:state}))[0]\n",
    "            \n",
    "        return action\n",
    "    \n",
    "    def run(self, state, reward, action, done):\n",
    "        reward = max(self.min_reward, min(self.self.max_reward, reward))\n",
    "        \n",
    "        self.history.add(state)\n",
    "        self.memory.add(state, reward, action, done)\n",
    "        \n",
    "        if self.t > FLAGS.learn_start:\n",
    "            if self.t % FLAGS.train_frequency:\n",
    "                self.q_\n",
    "        \n",
    "        \n",
    "        # 调用sess.run运行图，生成一步的训练过程数据  \n",
    "        train_summary = sess.run(merge_summary, feed={})\n",
    "        # 调用train_writer的add_summary方法将训练过程以及训练步数保存  \n",
    "        train_writer.add_summary(train_summary, step)\n",
    "        \n",
    "        self.t += 1\n",
    "    \n",
    "    def __build_network(self):\n",
    "         \"\"\"\n",
    "        build the natural network\n",
    "        \"\"\"\n",
    "        # Create placeholders\n",
    "        with tf.name_scope('actor_inputs'):\n",
    "            self.X = tf.placeholder(tf.float32, \n",
    "                                    shape=[None, FLAGS.frame_width, FLAGS.frame_height, FLAGS.history_length],\n",
    "                                    name=\"states\")\n",
    "            self.Y = tf.placeholder(tf.float32, shape=(self.n_y, None), name=\"action\")\n",
    "            self.disc_norm_ep_reward = tf.placeholder(tf.float32, name=\"td_error\")\n",
    "\n",
    "        with tf.name_scope(\"conv1\"):\n",
    "            conv1 = tf.nn.con2d(self.X, 32, \n",
    "                                kernel_size=[8, 8], strides=[4, 4],\n",
    "                                padding=\"same\", activation=tf.nn.relu)\n",
    "            # pool1 = tf.layers.max_pooling2d(inputs=conv1, pool_size=[2,2], strides=2)\n",
    "        with tf.name_scope(\"conv2\"):\n",
    "            conv2 = tf.nn.conv2(conv1, 64,\n",
    "                               kernel_size=[4, 4], strides=[2, 2],\n",
    "                               padding=\"same\", activation=tf.nn.relu)\n",
    "            # pool2 = tf.layers.max_pooling2d(inputs=conv2, pool_size=[2,2], strides=2)\n",
    "        with tf.name_scope(\"conv3\"):\n",
    "            conv3 = tf.nn.conv2(conv2, 64,\n",
    "                               kernel_size=[3, 3], strides=[1, 1],\n",
    "                               padding=\"same\", activation=tf.nn.relu)\n",
    "            # pool3 = tf.layers.max_pooling2d(inputs=conv3, pool_size=[2,2], strides=2)\n",
    "\n",
    "        with tf.name_scope(\"dense_layer\"):\n",
    "            conv3_flat = tf.reshape(conv3, [-1,  * 64])\n",
    "            dense1 = tf.layers.dense(conv3_flat, units=512, activation=tf.nn.relu)\n",
    "        \n",
    "        with tf.name_scope(\"logits_layer\"):\n",
    "            logits = tf.layers.dense(dense1, units=self.env.nA)\n",
    "            \n",
    "\n",
    "        with tf.name_scope('actor_loss'):\n",
    "            neg_log_prob = tf.nn.softmax_cross_entropy_with_logits_v2(logits=logits, labels=labels)\n",
    "            loss = tf.reduce_mean(neg_log_prob * self.disc_norm_ep_reward)  # reward guided loss\n",
    "\n",
    "        with tf.name_scope('actor_train'):\n",
    "            self.train_op = tf.train.AdamOptimizer(self.lr).minimize(loss)\n",
    "    \n",
    "    def __weigfht_variable(self, shape, name):\n",
    "        initial = tf.contrib.layers.xavier_initializer(seed=1)\n",
    "        return tf.get_variable(name, shape, initializer=initial)\n",
    "    \n",
    "    def __bias_bariable(self, shape, name):\n",
    "        initial = tf.contrib.layers.xavier_initializer(seed=1)\n",
    "        return tf.get_variable(name, shape, initializer=initial)\n",
    "    \n",
    "    def setup_summary(self):\n",
    "        # 生成准确率标量图 \n",
    "        average_reward = tf.placeholder('float32', None, name=\"average_reward\")\n",
    "        tf.summary.scalar(\"average_reward\", average_reward)\n",
    "        average_loss = tf.placeholder('float32', None, name=\"average_loss\")\n",
    "        tf.summary.scalar(\"average_loss\", average_loss)\n",
    "        average_q = tf.placeholder('float',None, name=\"average_q\")\n",
    "        tf.summary.scalar(\"average_q\", average_q)\n",
    "        \n",
    "        episode_max_reward = tf.placeholder('float',None, name=\"episode_max_reward\")\n",
    "        tf.summary.scalar(\"episode_max_reward\", episode_max_reward)\n",
    "        episode_min_reward = tf.placeholder('float',None, name=\"episode_min_reward\")\n",
    "        tf.summary.scalar(\"episode_min_reward\", episode_min_reward)\n",
    "        episode_avg_reward = tf.placeholder('float',None, name=\"episode_avg_reward\")\n",
    "        tf.summary.scalar(\"episode_avg_reward\", episode_avg_reward)\n",
    "        episode_num = tf.placeholder('float',None, name=\"episode_num\")\n",
    "        tf.summary.scalar(\"episode_num\", episode_num)\n",
    "        episode_learning_rate = tf.placeholder('float',None, name=\"episode_learning_rate\")\n",
    "        tf.summary.scalar(\"episode_learning_rate\", episode_learning_rate)\n",
    "        \n",
    "        # 定义一个写入summary的目标文件，dir为写入文件地址 \n",
    "        merge_summary = tf.summary.merge_all()\n",
    "        train_writer = tf.summary.FileWriter('./logs/', self.sess.graph)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "ename": "IndentationError",
     "evalue": "expected an indented block (<ipython-input-27-02b88fac2b31>, line 5)",
     "output_type": "error",
     "traceback": [
      "\u001b[1;36m  File \u001b[1;32m\"<ipython-input-27-02b88fac2b31>\"\u001b[1;36m, line \u001b[1;32m5\u001b[0m\n\u001b[1;33m    else:\u001b[0m\n\u001b[1;37m    ^\u001b[0m\n\u001b[1;31mIndentationError\u001b[0m\u001b[1;31m:\u001b[0m expected an indented block\n"
     ]
    }
   ],
   "source": [
    "def deep_Qlearning(env):\n",
    "    state_history = History()\n",
    "    state_memory = Memory()\n",
    "    env = Environment(gym.make('Breakout-v0'), state_history)\n",
    "    agent = Agent(env, state_history, state_memory)\n",
    "\n",
    "    if FLAGS.is_train:\n",
    "        # for trainning the deep Q learning model\n",
    "        max_avg_reward = 0\n",
    "        \n",
    "        for _ in range(FLAGS.episode_in_trains):\n",
    "            total_reward, total_loss, total_q, ep_reward = 0, 0, 0, 0\n",
    "            rewards, actions = [], []\n",
    "            state_history = env.random_start()\n",
    "            \n",
    "            for t in itertools.count():\n",
    "                # predict\n",
    "                action = agent.predict(state_history.get())\n",
    "                # action\n",
    "                state, reward, done = env.step(action)\n",
    "                # record\n",
    "                # target = reward + gamma * np.amax(model.predict(next_state))\n",
    "                agent.run(state, reward, action, done)\n",
    "                \n",
    "                if done:\n",
    "                    ep_reward = 0\n",
    "                    rewards.append(ep_reward)\n",
    "                    state_history = env.random_start()\n",
    "                else:\n",
    "                    ep_reward += reward\n",
    "                \n",
    "                actions.append(action)\n",
    "                total_reward += reward            \n",
    "    else:\n",
    "        # for test the deep Q learning model\n",
    "        best_reward, best_idx = 0, 0\n",
    "        for _ in range(FLAGS.episode_in_test):\n",
    "            state_history = env.random_start()\n",
    "            current_reward = 0\n",
    "            \n",
    "            for t in itertools.count():\n",
    "                # predict\n",
    "                action = agent.predict(state_history.get())\n",
    "                # action\n",
    "                state, reward, done = env.step(action)\n",
    "                # record\n",
    "                state_history.push(state)\n",
    "                \n",
    "                current_reward += reward\n",
    "                if done: break\n",
    "            \n",
    "            # print out the reward \n",
    "            if current_reward > best_reward:\n",
    "                best_reward = current_reward\n",
    "                best_idx = _\n",
    "                print(\"*\"*80)\n",
    "                print(\"[{}] Best reward:{}\".format(best_idx, best_reward))\n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "'int' object is not subscriptable",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-12-0ec9e5ef6dec>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mrandom\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrandrange\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m5\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;31mTypeError\u001b[0m: 'int' object is not subscriptable"
     ]
    }
   ],
   "source": [
    "random.randrange(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
